import time

import scrapy
from ..items import CbcItem
from selenium import webdriver
from selenium.webdriver import FirefoxOptions
from threading import Thread, Lock
class ChinaSpider(scrapy.Spider):
    name = 'nationalpost'
    allowed_domains = ['https://nationalpost.com/']
    start_urls = ['https://nationalpost.com/search/?search_text=China&date_range=-365d&sort=score']
    count = 8661;
    def __init__(self):
        # 在初始化时，创建driver
        super(ChinaSpider, self).__init__(name='nationalpost')
        option = FirefoxOptions()
        option.headless = True
        self.driver = webdriver.Firefox(options=option)

    def parse(self, response):

        tr_list = response.xpath("//div[@class='article-card__details']")
        tr = tr_list[0]
        item = CbcItem()
        tr.xpath(
            "./div[@class='article-card__meta-bottom']/span/text()").extract_first()
        href = tr.xpath("./a/@href").extract_first()
        item["title"] = tr.xpath("./a/h3/@title").extract_first()
        item["brief"] = tr.xpath(
            "./a/p/text()").extract_first()
        item['herf'] = 'https://nationalpost.com' + href
        yield scrapy.Request(
            item['herf'],
            callback=self.parse_detail,
            meta={"item": item},
            dont_filter=True
        )
        self.count=self.count+20
        if(self.count<=9990):
            print(self.count)
            yield scrapy.Request(
                "https://nationalpost.com/search/?search_text=China&date_range=-365d&sort=desc&from="+str(self.count),
                callback=self.parse,
                dont_filter=True
            )


    def parse_detail(self, response):
        item = response.meta['item']
        item["content"] = response.xpath("//*[@id='content']/div[1]/div[1]/div[3]/article/p/text()").extract()
        item["content"] = "".join(item["content"])
        item["publish_date"] = response.xpath("//*[@id='article-byline']/div[2]/div[2]/div/div[1]/span").extract_first()
        yield(item)